import pandas as pd
# Enable copy-on-write optimizations to save on memory and avoid making copies of data until written to
# The line below is to stop seeing the "SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame"
pd.options.mode.copy_on_write = True
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
import numpy as np
import time

# Load the raw dataset
df_raw = pd.read_csv('mgs_dataset.csv')

# Display the first few rows of the raw dataframe to inspect its structure and data types
df_raw.head()

# Convert UNIX timestamps to datetime format for better readability
df_raw['timestamp_created'] = pd.to_datetime(df_raw['timestamp_created'], unit='s')
df_raw['timestamp_updated'] = pd.to_datetime(df_raw['timestamp_updated'], unit='s')

# Remove duplicate reviews based on 'author/steamid' and 'review' columns to ensure unique entries
df_raw.drop_duplicates(subset=['author/steamid', 'review'], inplace=True)

# Select only the necessary columns for analysis
df = df_raw[['author/steamid', 'author/playtime_at_review', 
             'timestamp_created', 'language', 'voted_up', 'review']]

# Rename columns for better readability and understanding
df.rename(columns={
    'author/steamid': 'steamid',
    'author/playtime_at_review': 'playtime_at_review',
    'timestamp_created': 'review_date',
    'voted_up' : 'recommendation'
}, inplace=True)

# Convert boolean values in 'recommendation' to string labels for clarity
df['recommendation'] = df['recommendation'].replace({True: 'recommended', False: 'not recommended'})

# Fill missing values in the 'review' column with empty strings
df['review'] = df['review'].fillna('')

# Save the cleaned dataset to a new CSV file for future use
df.to_csv('mgs_dataset_cleaned.csv', index=False)

# Display the first few rows of the cleaned dataframe to verify the changes
df.head()

recommendation_counts = df.recommendation.value_counts()
recommendation_percentages = df.recommendation.value_counts(normalize=True) * 100

# Combine the counts and percentages into a single DataFrame
recommendation_summary = pd.DataFrame({
    'Count': recommendation_counts,
    'Percentage (%)': recommendation_percentages
})

print(recommendation_summary)

                 Count  Percentage (%)
recommendation                        
recommended       1329       73.833333
not recommended    471       26.166667

# Extract year and month for grouping
df['month_year'] = df['review_date'].dt.to_period('M')

# Count the recommendations per month
monthly_recommendation_counts = df.groupby(['month_year', 'recommendation']).size().unstack().fillna(0)

# Plot the stacked bar chart
ax = monthly_recommendation_counts.plot(kind='bar', stacked=True, figsize=(14, 7), color=['tab:red', 'tab:blue'])
ax.set_title('Monthly Stacked Bar Chart of Recommendations')
ax.set_ylabel('Count')
ax.set_xlabel('')
ax.set_xticklabels([f'{period.start_time.strftime("%b %Y")}' for period in monthly_recommendation_counts.index], rotation=45)
ax.legend(title='Recommendation')
ax.set_axisbelow(True)
plt.grid(axis='y')

# Annotate the counts on the bars
for p in ax.patches:
    height = p.get_height()
    if height > 0:
        ax.annotate(f'{int(height)}', (p.get_x() + p.get_width() / 2., p.get_y() + height / 2.), ha='center', va='center', xytext=(0, 0), textcoords='offset points', fontsize=9, color='black', weight='bold')

plt.show()

# Create a variable for playtime in hours
playhrs = df['playtime_at_review'] / 60
playhrs.describe()

count    1800.000000
mean       10.966444
std        13.693124
min         0.083333
25%         2.566667
50%         9.066667
75%        15.037500
max       291.366667
Name: playtime_at_review, dtype: float64

# Create the boxplot for playtime at review for recommended and not recommended reviews
plt.figure(figsize=(10, 6))
sns.boxplot(x='recommendation', y=playhrs, data=df)
plt.xlabel('Recommendation')
plt.ylabel('Playtime at Review (hours)')
plt.title('Playtime at Review for Recommended and Not Recommended Reviews')
plt.ylim(0,80)
plt.grid(axis='y')
plt.show()

# Review counts by language
review_counts_by_language = df['language'].value_counts()
print(review_counts_by_language)

language
english       1249
spanish        149
french          73
german          51
italian         48
japanese        38
brazilian       37
latam           32
russian         29
turkish         25
polish          23
czech            7
schinese         7
koreana          6
dutch            4
ukrainian        4
finnish          3
tchinese         3
hungarian        3
portuguese       3
swedish          2
danish           2
thai             1
norwegian        1
Name: count, dtype: int64

# Filter the dataset
df = df[df.language == 'english']

# Ensure nltk vader lexicon are downloaded
# nltk.download('vader_lexicon')
from nltk.sentiment import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()

vader_start_time = time.time()

# Apply VADER to each review and create new columns for the scores
df['vader_neg'] = df['review'].apply(lambda x: sid.polarity_scores(x)['neg'])
df['vader_neu'] = df['review'].apply(lambda x: sid.polarity_scores(x)['neu'])
df['vader_pos'] = df['review'].apply(lambda x: sid.polarity_scores(x)['pos'])
df['vader_compound'] = df['review'].apply(lambda x: sid.polarity_scores(x)['compound'])

# Define sentiment based on compound score
def get_sentiment(score):
    if score >= 0.05:
        return 'Positive'
    elif score <= -0.05:
        return 'Negative'
    else:
        return 'Neutral'

df['vader_sentiment'] = df['vader_compound'].apply(get_sentiment)

vader_end_time = time.time()
vader_time = vader_end_time - vader_start_time

df.head()

# Display sentiment counts
vader_sentiment_counts = df['vader_sentiment'].value_counts()
print(f"Sentiment Distribution:\n{vader_sentiment_counts}")

Sentiment Distribution:
vader_sentiment
Positive    726
Negative    262
Neutral     261
Name: count, dtype: int64

from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline
from scipy.special import softmax

2024-07-31 17:22:18.516204: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2024-07-31 17:22:18.554144: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2024-07-31 17:22:18.562718: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2024-07-31 17:22:18.581730: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2024-07-31 17:22:21.185968: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT

# Load tokenizer, model, and the sentiment-analysis pipeline
tokenizer = AutoTokenizer.from_pretrained('cardiffnlp/twitter-roberta-base-sentiment')
model = AutoModelForSequenceClassification.from_pretrained('cardiffnlp/twitter-roberta-base-sentiment')
sentiment_pipeline = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer, padding=True, truncation=True, max_length=512)

roberta_start_time = time.time()

# Function to get sentiment scores
def get_roberta_sentiment(review):
    encoded_review = tokenizer(review, return_tensors='pt', padding=True, truncation=True, max_length=512)
    output = model(**encoded_review)
    
    logits = output[0][0].detach().numpy()  # Raw logits
    probabilities = softmax(logits)  # Softmax probabilities
    # Calculate compound-like score using weighted sum and tanh
    compound_score = np.tanh(probabilities[2] * 1 + probabilities[1] * 0 + probabilities[0] * -1) 
    
    result = sentiment_pipeline(review)[0]
    label_map = {
        'LABEL_0': 'Negative',
        'LABEL_1': 'Neutral',
        'LABEL_2': 'Positive'
    }
    sentiment_label = label_map[result['label']]

    return probabilities[0], probabilities[1], probabilities[2], compound_score, sentiment_label, result['score']

# Apply the RoBERTa model to the reviews
df['roberta_neg'], df['roberta_neu'], df['roberta_pos'], df['roberta_compound'], df['roberta_sentiment'], df['roberta_score'] = zip(*df['review'].apply(get_roberta_sentiment))

roberta_end_time = time.time()

roberta_time = roberta_end_time - roberta_start_time

df.head()

# Display sentiment counts
roberta_sentiment_counts = df['roberta_sentiment'].value_counts()
print(f"Sentiment Distribution:\n{roberta_sentiment_counts}")

Sentiment Distribution:
roberta_sentiment
Positive    550
Neutral     357
Negative    342
Name: count, dtype: int64

# Define the figure and axes
fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# Overall histogram
sns.histplot(data=df, x='roberta_score', kde=True, bins=20, color='tab:green', ax=axes[0, 0])
axes[0, 0].set_title('Histogram of Overall RoBERTa Confidence Scores')
axes[0, 0].set_xlabel('Confidence Score')
axes[0, 0].set_ylabel('Frequency')
axes[0, 0].set_xlim(0.35,1)

# Histogram for Positive sentiment
sns.histplot(data=df[df['roberta_sentiment'] == 'Positive'], x='roberta_score', kde=True, bins=20, color='tab:blue', ax=axes[0, 1])
axes[0, 1].set_title('Histogram of RoBERTa Confidence Scores for Positive Sentiment')
axes[0, 1].set_xlabel('Confidence Score')
axes[0, 1].set_ylabel('Frequency')
axes[0, 1].set_xlim(0.35,1)

# Histogram for Neutral sentiment
sns.histplot(data=df[df['roberta_sentiment'] == 'Neutral'], x='roberta_score', kde=True, bins=20, color='tab:orange', ax=axes[1, 0])
axes[1, 0].set_title('Histogram of RoBERTa Confidence Scores for Neutral Sentiment')
axes[1, 0].set_xlabel('Confidence Score')
axes[1, 0].set_ylabel('Frequency')
axes[1, 0].set_xlim(0.35,1)

# Histogram for Negative sentiment
sns.histplot(data=df[df['roberta_sentiment'] == 'Negative'], x='roberta_score', kde=True, bins=20, color='tab:red', ax=axes[1, 1])
axes[1, 1].set_title('Histogram of RoBERTa Confidence Scores for Negative Sentiment')
axes[1, 1].set_xlabel('Confidence Score')
axes[1, 1].set_ylabel('Frequency')
axes[1, 1].set_xlim(0.35,1)

# Adjust layout
plt.tight_layout()
plt.show()

print(f"VADER process took {vader_time:.2f} seconds")
print(f"RoBERTa process took {roberta_time:.2f} seconds")

VADER process took 2.74 seconds
RoBERTa process took 315.40 seconds

from sklearn.metrics import cohen_kappa_score

# Function to create a donut chart with title inside the donut
def create_donut_chart(ax, data, labels, title, colors):
    wedges, texts, autotexts = ax.pie(data, labels=labels, colors=colors, autopct='%1.1f%%',
                                     startangle=0, pctdistance=0.85, wedgeprops=dict(width=0.3))
    for text in texts:
        text.set_fontsize(12)
    for autotext in autotexts:
        autotext.set_fontsize(12)
        autotext.set_color('black')
    centre_circle = plt.Circle((0,0),0.70,fc='white')
    ax.add_artist(centre_circle)
    # Add the title inside the donut
    ax.text(0, 0, title, horizontalalignment='center', verticalalignment='center', fontsize=18, fontweight='bold')
    ax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.

# Prepare data for Steam recommendation
steam_recommendation_counts = df['recommendation'].value_counts()
steam_labels = ['not recommended', 'recommended']
steam_data = [steam_recommendation_counts.get(label, 0) for label in steam_labels]
steam_colors = ['#ff9999', '#66b3ff']  # Red for not recommended, blue for recommended

# Prepare data for VADER sentiment
vader_sentiment_counts = df['vader_sentiment'].value_counts()
vader_labels = ['Negative', 'Neutral', 'Positive']
vader_data = [vader_sentiment_counts.get(label, 0) for label in vader_labels]
vader_colors = ['#ff9999', '#ffcc99', '#66b3ff']  # Red for negative, light orange for neutral, blue for positive

# Prepare data for RoBERTa sentiment
roberta_sentiment_counts = df['roberta_sentiment'].value_counts()
roberta_labels = ['Negative', 'Neutral', 'Positive']
roberta_data = [roberta_sentiment_counts.get(label, 0) for label in roberta_labels]
roberta_colors = ['#ff9999', '#ffcc99', '#66b3ff']  # Red for negative, light orange for neutral, blue for positive

# Create subplots
fig, axs = plt.subplots(1, 3, figsize=(18, 6))

# Create donut charts
create_donut_chart(axs[0], steam_data, steam_labels, 'Steam\nRecommendations', steam_colors)
create_donut_chart(axs[1], vader_data, vader_labels, 'VADER\nSentiments', vader_colors)
create_donut_chart(axs[2], roberta_data, roberta_labels, 'RoBERTa\nSentiments', roberta_colors)

plt.tight_layout()
plt.show()

#Calculate Cohen's Kappa
kappa = cohen_kappa_score(df['vader_sentiment'], df['roberta_sentiment'])
print(f"Cohen's Kappa Score: {kappa}")

Cohen's Kappa Score: 0.5261585814790576

# Calculate average sentiment scores by month for VADER and RoBERTa
monthly_sentiments = df.groupby('month_year').agg({
    'vader_compound': 'mean',
    'roberta_compound': 'mean'
}).reset_index()

# Convert month_year back to datetime for plotting
monthly_sentiments['month_year'] = monthly_sentiments['month_year'].dt.to_timestamp()

# Plot average sentiment scores by month for VADER and RoBERTa
plt.figure(figsize=(14, 7))
plt.plot(monthly_sentiments['month_year'], monthly_sentiments['vader_compound'], label='VADER Compound Score', color='tab:green')
plt.plot(monthly_sentiments['month_year'], monthly_sentiments['roberta_compound'], label='RoBERTa Compound Score', color='tab:orange')
plt.xlabel('Month')
plt.ylabel('Average Sentiment Score')
plt.title('Average Monthly Sentiment Scores Over Time (VADER and RoBERTa)')
plt.legend()
plt.grid(True)
plt.xticks(rotation=45)
plt.xticks(monthly_sentiments['month_year'], [date.strftime('%b %Y') for date in monthly_sentiments['month_year']], rotation=45)
plt.show()

# Calculate the sum and count of sentiment scores by month for VADER and RoBERTa
monthly_sentiments = df.groupby('month_year').agg({
    'vader_compound': ['sum', 'count'],
    'roberta_compound': ['sum', 'count']
}).reset_index()

# Flatten the column multi-index
monthly_sentiments.columns = ['month_year', 'vader_sum', 'vader_count', 'roberta_sum', 'roberta_count']

# Calculate the cumulative sum and count of sentiment scores
monthly_sentiments['vader_cumulative_sum'] = monthly_sentiments['vader_sum'].cumsum()
monthly_sentiments['vader_cumulative_count'] = monthly_sentiments['vader_count'].cumsum()
monthly_sentiments['roberta_cumulative_sum'] = monthly_sentiments['roberta_sum'].cumsum()
monthly_sentiments['roberta_cumulative_count'] = monthly_sentiments['roberta_count'].cumsum()

# Calculate the cumulative mean of sentiment scores
monthly_sentiments['vader_cumulative_mean'] = monthly_sentiments['vader_cumulative_sum'] / monthly_sentiments['vader_cumulative_count']
monthly_sentiments['roberta_cumulative_mean'] = monthly_sentiments['roberta_cumulative_sum'] / monthly_sentiments['roberta_cumulative_count']

# Convert month_year back to datetime for plotting
monthly_sentiments['month_year'] = monthly_sentiments['month_year'].dt.to_timestamp()

# Plot cumulative mean VADER and RoBERTa compound scores over time
plt.figure(figsize=(14, 7))
plt.plot(monthly_sentiments['month_year'], monthly_sentiments['vader_cumulative_mean'], label='VADER Cumulative Mean Score', color='tab:green')
plt.plot(monthly_sentiments['month_year'], monthly_sentiments['roberta_cumulative_mean'], label='RoBERTa Cumulative Mean Score', color='tab:orange')
plt.xlabel('Month')
plt.ylabel('Cumulative Mean Sentiment Score')
plt.title('Cumulative Mean Sentiment Scores Over Time')
plt.legend()
plt.grid(True)
plt.xticks(monthly_sentiments['month_year'], [date.strftime('%b %Y') for date in monthly_sentiments['month_year']], rotation=45)
plt.show()

# Melt the DataFrame to have a long format for seaborn's boxplot
df_melted_neg = df.melt(id_vars=['recommendation'], value_vars=['vader_neg', 'roberta_neg'], var_name='Sentiment_Score', value_name='Score')
df_melted_neu = df.melt(id_vars=['recommendation'], value_vars=['vader_neu', 'roberta_neu'], var_name='Sentiment_Score', value_name='Score')
df_melted_pos = df.melt(id_vars=['recommendation'], value_vars=['vader_pos', 'roberta_pos'], var_name='Sentiment_Score', value_name='Score')
df_melted_compound = df.melt(id_vars=['recommendation'], value_vars=['vader_compound', 'roberta_compound'], var_name='Sentiment_Score', value_name='Score')

# Create subplots
fig, axes = plt.subplots(2, 2, figsize=(18, 14))

# Plot for compound sentiment
sns.boxplot(ax=axes[0, 0], x='recommendation', y='Score', hue='Sentiment_Score', data=df_melted_compound, palette='Set2')
axes[0, 0].set_title('Compound Sentiment Scores by Recommendation (VADER and RoBERTa)')
axes[0, 0].set_xlabel('')
axes[0, 0].set_ylabel('Compound Score')
axes[0, 0].legend(title='Sentiment Score', bbox_to_anchor=(1.05, 1), loc='upper left')
axes[0, 0].grid(axis='y')
axes[0, 0].axvline(x=0.5, color='gray')

# Plot for positive sentiment
sns.boxplot(ax=axes[0, 1], x='recommendation', y='Score', hue='Sentiment_Score', data=df_melted_pos, palette='Set2')
axes[0, 1].set_title('Positive Sentiment Scores by Recommendation (VADER and RoBERTa)')
axes[0, 1].set_xlabel('')
axes[0, 1].set_ylabel('Positive Score')
axes[0, 1].legend(title='Sentiment Score', bbox_to_anchor=(1.05, 1), loc='upper left')
axes[0, 1].grid(axis='y')
axes[0, 1].axvline(x=0.5, color='gray')

# Plot for neutral sentiment
sns.boxplot(ax=axes[1, 0], x='recommendation', y='Score', hue='Sentiment_Score', data=df_melted_neu, palette='Set2')
axes[1, 0].set_title('Neutral Sentiment Scores by Recommendation (VADER and RoBERTa)')
axes[1, 0].set_xlabel('')
axes[1, 0].set_ylabel('Neutral Score')
axes[1, 0].legend(title='Sentiment Score', bbox_to_anchor=(1.05, 1), loc='upper left')
axes[1, 0].grid(axis='y')
axes[1, 0].axvline(x=0.5, color='gray')

# Plot for negative sentiment
sns.boxplot(ax=axes[1, 1], x='recommendation', y='Score', hue='Sentiment_Score', data=df_melted_neg, palette='Set2')
axes[1, 1].set_title('Negative Sentiment Scores by Recommendation (VADER and RoBERTa)')
axes[1, 1].set_xlabel('')
axes[1, 1].set_ylabel('Negative Score')
axes[1, 1].legend(title='Sentiment Score', bbox_to_anchor=(1.05, 1), loc='upper left')
axes[1, 1].grid(axis='y')
axes[1, 1].axvline(x=0.5, color='gray')

# Adjust layout
plt.tight_layout()
plt.show()

# Create the density plot for VADER Compound Scores and RoBERTa Compound Scores
plt.figure(figsize=(10, 6))
sns.kdeplot(df['vader_compound'], fill=True, color='green', label='VADER Compound')
sns.kdeplot(df['roberta_compound'], fill=True, color='orange', label='RoBERTa Compound')
plt.title('VADER Compound Scores and RoBERTa Compound Scores')
plt.xlabel('Compound Score')
plt.ylabel('Density')
plt.legend()
plt.show()

# Combine and compare VADER on RoBERTa results on a pairplot
pairplot = sns.pairplot(df[['vader_compound', 'roberta_compound', 'vader_neg', 'vader_neu', 'vader_pos', 'roberta_neg', 'roberta_neu', 'roberta_pos', 'recommendation']],
                        hue='recommendation', palette={'recommended': 'tab:blue', 'not recommended': 'tab:red'}, diag_kind='kde')

pairplot.fig.suptitle('Pairplot of VADER and RoBERTa Sentiment Scores', y=1.02)
pairplot.savefig('pairplot_vader_roberta_sentiment_scores.png')
plt.show()

# Convert Steam recommendations to binary values for correlation analysis
df['recommendation_binary'] = df['recommendation'].map({'recommended': 1, 'not recommended': 0})

# Calculate Pearson correlation coefficients
pearson_correlations = df[['vader_compound', 'roberta_compound', 'vader_neg', 'roberta_neg', 'vader_neu', 'roberta_neu', 'vader_pos', 'roberta_pos', 'recommendation_binary']].corr(method='pearson')

# Calculate Spearman correlation coefficients
spearman_correlations = df[['vader_compound', 'roberta_compound', 'vader_neg', 'roberta_neg', 'vader_neu', 'roberta_neu', 'vader_pos', 'roberta_pos', 'recommendation_binary']].corr(method='spearman')

# Plot Pearson correlation heatmap
plt.figure(figsize=(12, 10))
sns.heatmap(pearson_correlations, annot=True, fmt=".2f", cmap="coolwarm", vmin=-1, vmax=1)
plt.title("Pearson Correlation Heatmap")
plt.savefig("Pearson.png")
plt.show()

# Plot Spearman correlation heatmap
plt.figure(figsize=(12, 10))
sns.heatmap(spearman_correlations, annot=True, fmt=".2f", cmap="coolwarm", vmin=-1, vmax=1)
plt.title("Spearman Correlation Heatmap")
plt.savefig("Spearman.png")
plt.show()

# Print a summary of key results
print("Key Correlation Coefficients (Pearson):")
print(pearson_correlations.loc[['vader_compound', 'roberta_compound'], ['vader_compound', 'roberta_compound', 'recommendation_binary']])
print("\nKey Correlation Coefficients (Spearman):")
print(spearman_correlations.loc[['vader_compound', 'roberta_compound'], ['vader_compound', 'roberta_compound', 'recommendation_binary']])

Key Correlation Coefficients (Pearson):
                  vader_compound  roberta_compound  recommendation_binary
vader_compound          1.000000          0.649418               0.348703
roberta_compound        0.649418          1.000000               0.605514

Key Correlation Coefficients (Spearman):
                  vader_compound  roberta_compound  recommendation_binary
vader_compound          1.000000          0.628814               0.307462
roberta_compound        0.628814          1.000000               0.588368

import re
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from gensim import corpora
from gensim.models.ldamodel import LdaModel
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis

# Ensure nltk stopwords and punkt tokenizer are downloaded
# nltk.download('stopwords')
# nltk.download('punkt')
# Combine all reviews into a single list for LDA
reviews = df['review'].values

reviews
# Preprocess text (cleaning and tokenization)
def preprocess_text(text):
    text = text.lower()  # Lowercase
    text = re.sub(r'\s+', ' ', text)  # Remove extra whitespace
    text = re.sub(r'\W', ' ', text)  # Remove non-word characters
    tokens = word_tokenize(text)  # Tokenize
    tokens = [word for word in tokens if word not in stopwords.words('english')]  # Remove stopwords
    return tokens

# Apply preprocessing to all reviews
processed_reviews = [preprocess_text(review) for review in reviews]

# Create dictionary and corpus
dictionary = corpora.Dictionary(processed_reviews)
corpus = [dictionary.doc2bow(review) for review in processed_reviews]

# Save dictionary and corpus to files
dictionary.save('lda_dictionary.dict')
corpora.MmCorpus.serialize('lda_corpus.mm', corpus)

# Train LDA model
num_topics = 5  # Number of topics
lda_model = LdaModel(corpus, num_topics=num_topics, id2word=dictionary, passes=15)

# Save the LDA model
lda_model.save('lda_model.model')

# Visualize the topics using pyLDAvis
lda_vis = gensimvis.prepare(lda_model, corpus, dictionary)
pyLDAvis.save_html(lda_vis, 'lda_visualization.html')

# To display in Jupyter Notebook
pyLDAvis.display(lda_vis)

huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
/home/pndgr/jupyter-env/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=38585) is multi-threaded, use of fork() may lead to deadlocks in the child.
  pid = os.fork()
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)

from wordcloud import WordCloud

# Filter the reviews based on recommendations
recommended_reviews = df[df['recommendation'] == 'recommended']['review'].values
not_recommended_reviews = df[df['recommendation'] == 'not recommended']['review'].values

# Combine all reviews into a single string
recommended_text = ' '.join(recommended_reviews)
not_recommended_text = ' '.join(not_recommended_reviews)

# Define a function to preprocess text
def preprocess_text(text, additional_stopwords):
    text = text.lower()  # Lowercase
    text = re.sub(r'\s+', ' ', text)  # Remove extra whitespace
    text = re.sub(r'\W', ' ', text)  # Remove non-word characters
    tokens = word_tokenize(text)  # Tokenize
    stop_words = set(stopwords.words('english')) | set(additional_stopwords)
    tokens = [word for word in tokens if word not in stop_words]  # Remove stopwords
    return ' '.join(tokens)

# Define additional stopwords
additional_stopwords = {'game', 'games'}

# Preprocess the text
recommended_text_cleaned = preprocess_text(recommended_text, additional_stopwords)
not_recommended_text_cleaned = preprocess_text(not_recommended_text, additional_stopwords)

# Path to a TrueType font file (You may need to adjust the path to a TrueType font file on your system)
font_path = '/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf'

# Generate word clouds
def generate_wordcloud(text, title, color, font_path):
    wordcloud = WordCloud(width=800, height=400, background_color=color, colormap='viridis', font_path=font_path).generate(text)
    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.title(title, size=20)
    plt.show()

# Word cloud for recommended reviews
generate_wordcloud(recommended_text_cleaned, 'Word Cloud for Recommended Reviews', 'white', font_path)

# Word cloud for not recommended reviews
generate_wordcloud(not_recommended_text_cleaned, 'Word Cloud for Not Recommended Reviews', 'white', font_path)

	author/last_played	author/num_games_owned	author/num_reviews	author/playtime_at_review	author/playtime_forever	author/playtime_last_two_weeks	author/steamid	hidden_in_steam_china	language	...	review	steam_china_location	steam_purchase	timestamp_created	timestamp_updated	voted_up	written_during_early_access
0	1701413897	181	13	882	882	0	76561198087178952	True	english	...	It's a decent port, but it could be better.	NaN	True	1721789799	1721789799	True	False
1	1721780706	203	17	771	771	771	76561198798779983	True	english	...	The best Quentin Tarantino film that I don't w...	NaN	False	1721781822	1721781822	True	False
2	1721793868	0	27	1559	1935	1177	76561198169484181	True	french	...	Champion olympique des Olympiades esquimaux in...	NaN	True	1721769431	1721769431	True	False
3	1721744655	0	59	635	635	635	76561198069601045	True	english	...	Undoubtedly one of the most influential video ...	NaN	True	1721744808	1721745044	True	False
4	1721806317	162	97	247	727	650	76561198155945496	True	english	...	ta bueno	NaN	True	1721734612	1721734612	True	False

	steamid	playtime_at_review	review_date	language	recommendation	review
0	76561198087178952	882	2024-07-24 02:56:39	english	recommended	It's a decent port, but it could be better.
1	76561198798779983	771	2024-07-24 00:43:42	english	recommended	The best Quentin Tarantino film that I don't w...
2	76561198169484181	1559	2024-07-23 21:17:11	french	recommended	Champion olympique des Olympiades esquimaux in...
3	76561198069601045	635	2024-07-23 14:26:48	english	recommended	Undoubtedly one of the most influential video ...
4	76561198155945496	247	2024-07-23 11:36:52	english	recommended	ta bueno

	steamid	playtime_at_review	review_date	language	recommendation	review	month_year	vader_neg	vader_neu	vader_pos	vader_compound	vader_sentiment	roberta_neg	roberta_neu	roberta_pos	roberta_compound	roberta_sentiment	roberta_score
0	76561198087178952	882	2024-07-24 02:56:39	english	recommended	It's a decent port, but it could be better.	2024-07	0.000	0.645	0.355	0.5927	Positive	0.143172	0.386522	0.470306	0.315943	Positive	0.470306
1	76561198798779983	771	2024-07-24 00:43:42	english	recommended	The best Quentin Tarantino film that I don't w...	2024-07	0.085	0.624	0.291	0.6096	Positive	0.240758	0.329828	0.429413	0.186448	Positive	0.429413
3	76561198069601045	635	2024-07-23 14:26:48	english	recommended	Undoubtedly one of the most influential video ...	2024-07	0.000	0.813	0.187	0.7943	Positive	0.005146	0.046832	0.948022	0.736541	Positive	0.948022
4	76561198155945496	247	2024-07-23 11:36:52	english	recommended	ta bueno	2024-07	0.000	1.000	0.000	0.0000	Neutral	0.207345	0.610191	0.182464	-0.024876	Neutral	0.610191
5	76561198160647826	1100	2024-07-23 04:37:09	english	recommended	solid snake smoked 500 cigarettes	2024-07	0.000	0.714	0.286	0.1531	Positive	0.799420	0.191205	0.009375	-0.658434	Negative	0.799420

Import Libraries and Raw Data¶

Data Wrangling¶

Quick EDA¶

Recommendations Count¶

Playtime at Review¶

Language Review Count¶

Natural Language Processing¶

VADER¶

RoBERTa¶

Comparison and Analysis¶

Execution Time Comparison¶

Steam Recommendation and Sentiment Comparison¶

Sentiment Over Time¶

Average Monthly Sentiment Score¶

Cumulative Mean Monthly Sentiment Score¶

Sentiment Score Comparison¶

Compound Score Density¶

Sentiment Score Pairplot¶

Sentiment Score Correlation¶

LDA Topic Modelling¶

WordCloud¶